import threading
import Queue
import socket
#import sqlite3
from pysqlite2 import dbapi2 as sqlite
import re
import random
import time
 
socket.setdefaulttimeout(30)
resultfind = re.compile("No match for", re.IGNORECASE)
cachecommit = []

class PThread(threading.Thread):
    def __init__(self, pqueue, cqueue):
        threading.Thread.__init__(self)
        self.pqueue = pqueue
        self.cqueue = cqueue
        
    def run(self):
        global resultfind
        result = ""
        #retry = False
        while True:
            row = self.pqueue.get()
            if row == None:
                break
            domain = row[0] + '.com'
            try:
                s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                s.connect(("whois.hichina.com", 43))
                s.send(domain + "\r\n")
                result = s.recv(4096)
                s.close()
                used = not bool(resultfind.search(result))
                self.cqueue.put((True, used, row[0]))
            except:
                pass
                 #if retry > 2:
                         #break
                     #else:
                         #time.sleep(random.random())
                         #retry += 1
                         #print "%s Rtry %d" % (domain,retry)
                         #continue
            self.pqueue.task_done()

class CThread(threading.Thread):
    def __init__(self, cqueue, dbLock, dbcx):
        threading.Thread.__init__(self)
        self.cqueue = cqueue
        self.dbLock = dbLock
        self.dbcx = dbcx
        #self.dbcu = self.dbcx.cursor()

    def run(self):
        global cachecommit
        while True:
            row = self.cqueue.get()
            if row == None:
                break
            #self.dbLock.acquire()
            cachecommit.append(row)
            #print len(cachecommit)
            if len(cachecommit) > 5000:
                for tmp in cachecommit:
                    self.dbcx.execute("UPDATE char SET processed = ?, used = ? WHERE domain = ?", tmp)
                    print tmp[2] + ".com commited"
                self.dbcx.commit()
                del cachecommit
                cachecommit = []
#            print row[2], ".com commited"
#            self.dbcu.execute("UPDATE char SET processed = ?, used = ? WHERE domain = ?", row)
            #self.dbcx.commit()
            #self.dbLock.release()
            self.cqueue.task_done()
             
def main():
    cx = sqlite.connect("domains.db", check_same_thread=False)#, isolation_level=None)
    cx.execute("PRAGMA cache_size = 8000")
    #cx.execute("PRAGMA count_changes = 1")
    cx.execute("PRAGMA synchronous = OFF")
    cx.execute("PRAGMA temp_store = MEMORY")
    #cu = cx.cursor()
    #rows = cu.fetchmany(50000)
    #cu.close()
#    cu.close()
#    cx.close()
#    del cu
#    del cx
    pqueue = Queue.Queue()
    cqueue = Queue.Queue()
    dbLock = threading.RLock()
    for i in range(100):
        thp = PThread( pqueue, cqueue )
        thp.setDaemon(True)
        thp.start()


#   for i in range(5):
#        thc = CThread( cqueue, dbLock, cx )
#        thc.setDaemon(True)
#        thc.start()
    thc = CThread( cqueue, dbLock, cx )
    thc.setDaemon(True)
    thc.start()

    while True:
        rows = cx.execute("select domain from char where processed = 0")
        row = rows.fetchone()
        if row != None:
            pqueue.put(row)
            for row in rows.fetchmany(50000):
                pqueue.put(row)
            del rows
        else:
            break
        pqueue.join()
        cqueue.join()

    global cachecommit
    for tmp in cachecommit:
        cx.execute("UPDATE char SET processed = ?, used = ? WHERE domain = ?", tmp)
        print tmp[2] + ".com commited"
    cx.commit()
    cx.close()

main()
